#!/bin/bash

echo -e "\n" >> ${0}_`date +%F`.log
# 故障模拟时间(单位:s/请输入10的倍数)
test_time=10

zookeeper_port=16309
dbscale_port=16310

process_install_dir="/data"
#UID服务运行用户
process_uid=mysql

function process_filter() {
  #ps -u $process_uid -f | grep -v grep | grep -v $0 | grep $process_name
  ps -aux | grep -v grep | grep -v $0 | awk "/^${process_uid}/" | grep $process_name
}
function info(){
  echo -e "\e[1;32m$*\e[0m"
}
function time_log() {
  info "`date +%T` $1" | tee -a ${0}_`date +%F`.log
}
#检查进程和端口
function check_service() {

if [ `ss -tnl | grep $process_port |wc -l` -ne 0 ] && [ `process_filter | wc -l` -eq  $process_sum ];then
  time_log "$process_name $1"
  return 200
else
  time_log "$process_name $2"
  return 100
fi
}
#启动检查
function start_check() {
time_log "cmd: $process_start_cmd"
echo "$process_start_cmd" | bash
time_log "${process_name}启动检测中。。。"
if [ $? -eq 0 ];then
  while true
  do
     sleep 3s
     check_service "服务已启动,端口进程正常。。。" "服务正在启动中。。。"
    [ $? -eq 200  ] && break
  done
else
  time_log "服务启动检查失败!!!"
fi
}
#启动服务
function process_variables() {
case $process_name in
	dbscale) 
	      process_port=$dbscale_port
	      process_start_cmd="cd ${process_install_dir}/app/dbscale && nohup ./daemon/dbscale_daemon.py > ${process_install_dir}/dbscalelog/out.log 2>&1 &"
	      #paused_process_pid=`process_filter | grep 'config-file dbscale.conf' | awk '{print $2}'`
	      paused_process_pid=`process_filter | grep dbscale | awk '{print $2}' | xargs`
	;;
 	mysqld|mysql)
	process_name=mysqld
	      #process_port=$mysql_port
	      process_port=``
        mysql_data_dir="${process_install_dir}/mysqldata/`ls -rtl ${process_install_dir}/mysqldata/ |tail -1 |  awk '{print $NF}'`"
	      mysql_cmd=`ps -ef | grep -v grep | grep mysqld | sed -rn 's#(.*) /bin/sh (.*)#\2#gp'`
	      mysql_logfile=`ps -ef | grep -v grep | grep mysqld | sed -rn 's#(.*) /bin/sh (.*)#\2#gp' | xargs -n1 | awk -F "=" '/defaults-file/ {print $2}' | sed -rn 's#(.*)my.*conf#\1logfile#p'`
	      process_start_cmd="nohup  ${mysql_cmd} >${mysql_data_dir}/out.log 2>&1 &"
        # 宕mysqld时用这个参数
        # paused_process_pid=`process_filter | grep mysql.sock  | awk '{print $2}'`
        # 宕mysqld和mysqlsafe时用这个参数
        paused_process_pid=`process_filter | grep mysqld  | awk '{print $2}' | xargs `
	;;
	zookeeper)
	      process_port=$zookeeper_port
	      #process_bin_dir="${process_install_dir}/`process_filter | sed -rn 's#.*/app/(.*)/bin/.*#app/\1/bin#p'`"
	      #process_start_cmd="${process_bin_dir}/zkServer.sh start"
	      #paused_process_pid=`process_filter | awk '{print $2}'`
	      process_start_cmd="sudo systemctl start configuration"
	      paused_process_pid=`process_filter | awk '{print $2}'`
  ;;
esac
}
#进程故障
function process_fault() {
  kill_process_num=$(process_filter | awk '{print $2}' | xargs )
  time_log "cmd: kill -9 ${kill_process_num}"
  kill -9 "${kill_process_num}"
  sleep 3s
  if [ `process_filter |wc -l` == 0 ];then
    time_log "进程已杀死!"
    time_log "cmd: ps -aux | grep -v grep | grep -v $0 | awk "/^${process_uid}/" | grep $process_name\n"
    process_filter
    time_log "cmd: sleep ${test_time}s"
    sleep_cmd
    start_check 
  else
    curr_process_num=$(process_filter | awk '{print $2}' | xargs )
    if [[ "${curr_process_num}" == "${kill_process_num}" ]];then
        time_log "cmd: kill -9 ${kill_process_num} 执行失败,进程存活!" 
    else
        time_log "cmd: kill -9 ${kill_process_num}  执行成功,进程自动拉起!" 
        process_filter
        time_log "cmd: sleep ${test_time}s"
        sleep_cmd
    fi
  fi
}
#进程挂起
function process_pause() {
    time_log "cmd: kill -19 $paused_process_pid\n"
    kill -19 $paused_process_pid 
    time_log "cmd: ps -aux | grep -v grep | grep -v $0 | awk "/^${process_uid}/" | grep $process_name "
    process_filter | tee -a ${0}_`date +%F`.log
    if [[ `process_filter 2>/dev/null | awk '{print $8}' | tail -1` == T* ]];then
#      if [ $process_name != 'dbscale' ];then
        time_log  "$process_name进程$paused_process_pid已暂停"
        time_log "cmd: sleep ${test_time}s"
	      sleep_cmd
        time_log "cmd: kill -18 $paused_process_pid"
        kill -18 $paused_process_pid
	      for i in `seq 5`
	      do
	      sleep 1
            if [[ `process_filter | grep $paused_process_pid 2>/dev/null | awk '{print $8}' | tail -1` == S* ]];then
              time_log "cmd: kill -18 $paused_process_pid 执行成功,进程已恢复!\ncmd: ps -aux | grep -v grep | grep -v $0 | awk '/^${process_uid}/' | grep $process_name"
              process_filter
              exit
            else
	            time_log "cmd: cmd: kill -18 $paused_process_pid 进程恢复中。。。!"
	          fi
        done
	      echo "5s服务未恢复,请手动检测$process_name"
#      else
#        for count in `seq 15`
#        do
#	         new_paused_process_pid=`process_filter | grep 'config-file dbscale.conf' | awk '{print $2}'`
#	        if [[ "Sl" =~ `process_filter | grep $new_paused_process_pid 2>/dev/null | awk '{print $8}' | tail -1` ]] &&  [ `echo ${#new_paused_process_pid}` -ne 0 ];then
#                  time_log "$process_name 进程已自动恢复!\ncmd: ps -aux | grep -v grep | grep -v $0 | awk "/^${process_uid}/" | grep $process_name | grep $new_paused_process_pid"
#	          sleep 3s
#	          process_filter | grep $new_paused_process_pid | tee -a ${0}_`date +%F`.log
#	          exit
#	        else
#	          sleep 3s
#	          [ `expr $count % 3` -eq 0 ] && time_log "$process_name服务检测中。。。"
#	        fi
#        done
#        time_log "45秒未恢复,请检查服务!"
#	      process_filter | grep $new_paused_process_pid
#	      kill -18 $new_paused_process_pid
#      fi
    else
      time_log "cmd: kill -19 $paused_process_pid 执行失败,请检查服务!"
    fi
}
function sleep_cmd() {
#  time_log "\ncmd: sleep 10s"
  for i in `seq 5`
  do
    echo "current_time: `date +%T`" 
    sleep $[test_time/5]
  done

}


if [ $# != 2 ];then
   echo -e  "USAGE: dbscale|mysqld|zookeeper 9/19\nEXAMPLES:\n \t杀dbscale进程:   sh $0 dbscale 9\n \t挂起dbscale进程: sh $0 dbscale 19"
   exit
fi

process_name=$1
#process_name=mysql

begin_time=`date +%T`
echo -e "----------------------------------\n执行 $1 $2"
process_variables

time_log "cmd: ps -ef | grep -v grep | grep -v $0| awk "/^${process_uid}/" | grep $process_name"
process_filter | tee -a ${0}_`date +%F`.log
read -p "请确认进程是否正确,请确保除测试服务外无其他进程（y/n):" choose
#choose=y
if [ $choose == 'y' ];then
  process_sum=`process_filter | wc -l`
else
  exit
fi
case $2 in
#case $key in
 	      9)
         echo "10秒后执行cmd: kill -9 `process_filter | awk '{print $2}' | xargs -n2`"
	       time_log "cmd: sleep 10"
	       sleep 10
         process_fault
         time_log "cmd: ps -aux | grep -v grep | grep -v $0 | awk "/^${process_uid}/" | grep $process_name"
         process_filter | tee -a ${0}_`date +%F`.log
 	      ;;
      	19)
 	       echo paused_process_pid:$paused_process_pid
	       echo "10秒后执行cmd: kill -19 `echo $paused_process_pid`"
	       time_log "cmd: sleep 10"
	       sleep 10
         process_pause
        ;;
        *)
         echo "请遵照提示使用脚本。。。"
 esac
